{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import sys\n",
    "import time\n",
    "import _init_paths\n",
    "from core.config import cfg, merge_cfg_from_file, merge_cfg_from_list, assert_and_infer_cfg\n",
    "from core.test_engine import run_inference, get_inference_dataset, extend_results\n",
    "from datasets.json_dataset import JsonDataset\n",
    "import pickle\n",
    "import json\n",
    "import ipdb\n",
    "from pycocotools.cocoeval import COCOeval\n",
    "from datasets.json_dataset_evaluator import _log_detection_eval_metrics, _coco_bbox_results_one_category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def box_equal(box1, box2):\n",
    "    for i in range(len(box1)):\n",
    "        if abs(box1[i] - box2[i]) >= 0.01:\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "def dict_equal(dict1, dict2):\n",
    "    a =  dict1['image_id'] == dict2['image_id']\n",
    "    b = dict1['category_id'] == dict2['category_id']\n",
    "    c = dict1['score'] == dict2['score']\n",
    "    d  = box_equal(dict1['bbox'], dict2['bbox'])\n",
    "    return a&b&c&d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_file = '../Outputs/e2e_faster_rcnn_R-50-FPN_1x_vcoco/Nov26-09-29-09_wanbo_node35_step/test/bbox_vcoco_test_results.json'\n",
    "det_res = json.load(open(res_file, 'rb'))\n",
    "# res_file = '../Outputs/e2e_faster_rcnn_R-50-FPN_1x_vcoco/Nov26-09-29-09_wanbo_node35_step/test/bbox_vcoco_trainval_results.json'\n",
    "# det_res = json.load(open(res_file, 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "det_results = {}\n",
    "for det in det_res:\n",
    "    im_id = det['image_id']\n",
    "    tmp = det_results.setdefault(im_id, [])\n",
    "    tmp.append(np.concatenate([det['bbox'], [det['score'], det['category_id']]]))\n",
    "    det_results[im_id] = tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.boxes import xywh_to_xyxy, bbox_overlaps\n",
    "for k,v in det_results.items():\n",
    "    v = np.array(v).astype(np.float32)\n",
    "    trans = xywh_to_xyxy(v[:, :4])\n",
    "    v[:, :4] = trans\n",
    "    det_results[k] = v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_path = '../data/cache/vcoco_test_precomp_boxes.json'\n",
    "# save_path = '../data/cache/vcoco_trainval_precomp_boxes.json'\n",
    "for k,v in det_results.items():\n",
    "    det_results[k] = v.tolist()\n",
    "    \n",
    "with open(save_path, 'w') as f:\n",
    "    json.dump(det_results, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=1.22s)\n",
      "creating index...\n",
      "index created!\n",
      "loading vcoco annotations...\n",
      "add precomp box from /root/project/InteractNet/data/cache/vcoco_test_precomp_boxes.json\n"
     ]
    }
   ],
   "source": [
    "cfg.TEST.DATASETS = ('vcoco_test',)\n",
    "cfg.MODEL.NUM_CLASSES = 81\n",
    "cfg.TEST.PRECOMPUTED_PROPOSALS = False\n",
    "cfg.MODEL.VCOCO_ON = True\n",
    "cfg.VCOCO.USE_PRECOMP_BOX = True\n",
    "dataset_name, proposal_file = get_inference_dataset(0, is_parent=False)\n",
    "json_dataset = JsonDataset(dataset_name)\n",
    "vcocodb = json_dataset.get_roidb(gt=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ican_file = '../ican/Test_Faster_RCNN_R-50-PFN_2x_VCOCO.pkl'\n",
    "# open(ican_file, 'rb')\n",
    "ican = pickle.load(open(ican_file, 'rb'), encoding='latin1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.boxes import xywh_to_xyxy, xyxy_to_xywh, bbox_overlaps\n",
    "ican_det_res = []\n",
    "for im_id, v in ican.items():\n",
    "    for item in v:\n",
    "        res = {}\n",
    "        res['image_id'] = im_id\n",
    "        res['category_id'] = item[-2]\n",
    "        box = xyxy_to_xywh(item[2][None, :])[0]\n",
    "        res['bbox'] = box.tolist()\n",
    "#         res['bbox'] = [box[0]*scale[1], box[1]*scale[0], box[2]*scale[1], box[3]*scale[0]]\n",
    "        res['score'] = item[-1].tolist()\n",
    "        ican_det_res.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[24.310546875, 65.66578674316406, 349.72113037109375, 350.07659912109375]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xyxy_to_xywh(ican[294][0][2][None, :])[0].tolist()\n",
    "# np.max(ican[294][0][5]) > 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "119061"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ican_det_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'image_id': 385029, 'category_id': 51, 'bbox': [217.9744873046875, 170.88516235351562, 28.596038818359375, 30.3131103515625], 'score': 0.06117446720600128}\n",
      "{'image_id': 294, 'category_id': 1, 'bbox': [20.025070190429688, 60.60774230957031, 344.08143615722656, 360.11997985839844], 'score': 0.9992368221282959}\n"
     ]
    }
   ],
   "source": [
    "print(ican_det_res[100])\n",
    "print(det_res[0])\n",
    "ican_res_file = '../ican/ican_vcoco_det_res.json'\n",
    "with open(ican_res_file, 'w') as f:\n",
    "    json.dump(ican_det_res, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading and preparing results...\n",
      "DONE (t=1.46s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=25.72s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=4.84s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.065\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.102\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.072\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.043\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.072\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.096\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.057\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.088\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.090\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.062\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.095\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.126\n"
     ]
    }
   ],
   "source": [
    "coco_dt = json_dataset.COCO.loadRes(ican_res_file)\n",
    "coco_eval = COCOeval(json_dataset.COCO, coco_dt, 'bbox')\n",
    "coco_eval.evaluate()\n",
    "coco_eval.accumulate()\n",
    "_log_detection_eval_metrics(json_dataset, coco_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=0.94s)\n",
      "creating index...\n",
      "index created!\n",
      "loading vcoco annotations...\n",
      "add precomp box from /root/project/InteractNet/data/cache/vcoco_test_precomp_boxes.json\n"
     ]
    }
   ],
   "source": [
    "cfg.TEST.DATASETS = ('vcoco_test',)\n",
    "cfg.MODEL.NUM_CLASSES = 81\n",
    "cfg.TEST.PRECOMPUTED_PROPOSALS = False\n",
    "cfg.MODEL.VCOCO_ON = True\n",
    "cfg.VCOCO.USE_PRECOMP_BOX = True\n",
    "dataset_name, proposal_file = get_inference_dataset(0, is_parent=False)\n",
    "json_dataset = JsonDataset(dataset_name)\n",
    "vcocodb = json_dataset.get_roidb(gt=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from core.test import im_detect_all, im_detect_bbox_precomp_box\n",
    "num_classes = cfg.MODEL.NUM_CLASSES\n",
    "num_images = len(vcocodb)\n",
    "all_boxes = [[[] for _ in range(num_images)] for _ in range(num_classes)]\n",
    "for i, entry in enumerate(vcocodb):\n",
    "    scores, boxes, cls_boxes_i = im_detect_bbox_precomp_box(entry, json_dataset.json_category_id_to_contiguous_id)\n",
    "    extend_results(i, all_boxes, cls_boxes_i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "for cls_ind, cls in enumerate(json_dataset.classes):\n",
    "    if cls == '__background__':\n",
    "        continue\n",
    "    if cls_ind >= len(all_boxes):\n",
    "        break\n",
    "    cat_id = json_dataset.category_to_id_map[cls]\n",
    "    results.extend(_coco_bbox_results_one_category(\n",
    "        json_dataset, all_boxes[cls_ind], cat_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "156926"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(results)\n",
    "len(det_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "for i in range(len(results)):\n",
    "    if not dict_equal(results[i], det_res[i]):\n",
    "        count += 1\n",
    "        print(results[i], filter_det_res[i])\n",
    "        ipdb.set_trace()\n",
    "#         break\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading and preparing results...\n",
      "DONE (t=0.95s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=38.82s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=5.76s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.288\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.486\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.302\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.177\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.337\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.422\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.271\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.407\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.422\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.271\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.475\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.594\n"
     ]
    }
   ],
   "source": [
    "from datasets.task_evaluation import json_dataset_evaluator\n",
    "coco_eval = json_dataset_evaluator.evaluate_boxes(\n",
    "    json_dataset, all_boxes, './', use_salt=False, cleanup=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "res_file = '../Outputs/e2e_interact_net_R-50-FPN_1x/Dec30-10-43-23_wanbo-node33_step/test/detections.pkl'\n",
    "det_res = pickle.load(open(res_file, 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6681"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(det_res['all_losses']['affinity_pos_num'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4666"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(det_res['all_losses']['affinity_tp_num'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6983984433468043"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "4666/6681"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "358788"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(det_res['all_losses']['affinity_neg_num'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "355983"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(det_res['all_losses']['affinity_tf_num'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.992182012776347"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "355983/358788"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
